
***Analysis Script for Gothreau & Haas, forthcoming in Journal of Experimental Political Science***
***A Replication and Extension of Willer et al. (2013) Overdoing Gender: A Test of the Masculine Overcompensation Thesis***

***Minimum Detectable Effect (MDE) Size Analysis#

*In the below minimum detectable effect size analysis, we show how the MDE changes across different sample sizes, include as vertical lines our realized sample sizes for women (grey line) and men (black line), and we include as points the observed effect for men for a given outcome in the original Willer et al. study at the sample size for men in that study. We use means and standard deviations as reported in the original Willer et al. study.#

clear all

*************************
*************************
*********Study 1*********
*************************
*************************

local power = 0.9	//Desired power
local alpha = 0.05	//Significance level

clear
set obs 41
gen n_total = 50 + (_n - 1) * 25
local nratio = 0.96571429	//Ratio of experimental group to control group (1=equal allocation)
local sd1 = 1.52
local sd2 = 1.60
local sd3 = 3.16
local sd4 = 10.63
local baseline1 = 2.65
local baseline2 = 2.77
local baseline3 = 4.84
local baseline4 = 20.68

foreach i in 1 2 3 4 {
    gen mde`i' = .
}

forvalues r = 1/41 {
    local N = n_total[`r']
    foreach i in 1 2 3 4 {
        local sd = `sd`i''
		local base = `baseline`i''
        power twomeans `base', n(`N') power(0.9) sd(`sd') nratio(`nratio') alpha(0.05)
        replace mde`i' = r(diff) in `r'
    }
}

* Observed effect points at Willer et al. men sample size*
gen obs_x1 = 51
gen obs_y1 = 0.99
gen obs_x2 = 51
gen obs_y2 = 1.26
gen obs_x3 = 51
gen obs_y3 = 1.72
gen obs_x4 = 51
gen obs_y4 = 7.32

*Total sample sizes for vertical lines (numbers taken from the DOV_GENDER variable provided by AmeriSpeak and thus differ marginally from those taken from the self-identification variable used elsewhere. Substantive conclusions are unchanged.)
gen men_line_x = 1032
gen men_line_y = 0
gen men_line_y2 = 1

gen women_line_x = 336
gen women_line_y = 0
gen women_line_y2 = 1

twoway ///
    (line mde1 n_total, lcolor(blue%50)   lpattern(solid)     lwidth(medthick)) ///
    (line mde2 n_total, lcolor(red%50)    lpattern(dash)      lwidth(medthick)) ///
    (line mde3 n_total, lcolor(green%50)  lpattern(dot)       lwidth(medthick)) ///
    (line mde4 n_total, lcolor(orange%50) lpattern(longdash)  lwidth(medthick)) ///
    (scatter obs_y1 obs_x1, mcolor(blue)   msymbol(circle)) ///
    (scatter obs_y2 obs_x2, mcolor(red)    msymbol(circle)) ///
    (scatter obs_y3 obs_x3, mcolor(green)  msymbol(circle)) ///
    (scatter obs_y4 obs_x4, mcolor(orange) msymbol(circle)) ///
    (line men_line_y men_line_x, lcolor(black)  lpattern(dash) lwidth(medthick)) ///
    (line women_line_y women_line_x, lcolor(gs10) lpattern(dash) lwidth(medthick)), ///
    title("Study 1: MDE Across Sample Sizes") ///
    ytitle("Minimum Detectable Effect") ///
    xtitle("Total Sample Size (Gender Threat + Control)") ///
    xlabel(100(100)1050, grid) ///
    ylabel(, grid) ///
    legend(order( ///
        1 "Iraq War MDE" ///
        2 "Homosexuality MDE" ///
        3 "SUV Desirability MDE" ///
        4 "SUV Pay MDE" ///
        5 "Original Iraq War Effect" ///
        6 "Original Homosexuality Effect" ///
        7 "Original SUV Desire Effect" ///
        8 "Original SUV Pay Effect" ///
        9 "Replication Sample (Men)" ///
        10 "Replication Sample (Women)") ///
        ring(0) position(2) region(margin(l=-170 r=15)) size(small)) ///
    xline(1032, lpattern(dash) lcolor(black) lwidth(medthick)) ///
    xline(336, lpattern(dash) lcolor(gs10) lwidth(medthick)) ///
    graphregion(color(white))
graph export "FigureB1.pdf", as(pdf) replace

*Next, we repeat the same exercise but using our study means and standard deviations (see note above about gender variable. We also do not use recoded variables for this analysis).

clear
set obs 41
gen n_total = 50 + (_n - 1) * 25
local sd1 = 1.9160874
local sd2 = 2.2220349
local sd3 = 2.8944466
local sd4 = 17.261395
local baseline1 = 3.3206107
local baseline2 = 3.9922027
local baseline3 = 4.316092
local baseline4 = 22.148077
local nratio1 = 0.96374046
local nratio2 = 0.97465887
local nratio3 = 0.96743295
local nratio4 = 0.96711799

foreach i in 1 2 3 4 {
    gen mde`i' = .
}

forvalues r = 1/41 {
    local N = n_total[`r']
    foreach i in 1 2 3 4 {
        local sd = `sd`i''
		local base = `baseline`i''
		local nratio = `nratio`i''
        power twomeans `base', n(`N') power(0.9) sd(`sd') nratio(`nratio') alpha(0.05)
        replace mde`i' = r(diff) in `r'
    }
}

* Observed effect points at Willer et al. men sample size*
gen obs_x1 = 51
gen obs_y1 = 0.99
gen obs_x2 = 51
gen obs_y2 = 1.26
gen obs_x3 = 51
gen obs_y3 = 1.72
gen obs_x4 = 51
gen obs_y4 = 7.32

* Total sample sizes for vertical lines
gen men_line_x = 1032
gen men_line_y = 0
gen men_line_y2 = 1

gen women_line_x = 336
gen women_line_y = 0
gen women_line_y2 = 1

twoway ///
    (line mde1 n_total, lcolor(blue%50)   lpattern(solid)     lwidth(medthick)) ///
    (line mde2 n_total, lcolor(red%50)    lpattern(dash)      lwidth(medthick)) ///
    (line mde3 n_total, lcolor(green%50)  lpattern(dot)       lwidth(medthick)) ///
    (line mde4 n_total, lcolor(orange%50) lpattern(longdash)  lwidth(medthick)) ///
    (scatter obs_y1 obs_x1, mcolor(blue)   msymbol(circle)) ///
    (scatter obs_y2 obs_x2, mcolor(red)    msymbol(circle)) ///
    (scatter obs_y3 obs_x3, mcolor(green)  msymbol(circle)) ///
    (scatter obs_y4 obs_x4, mcolor(orange) msymbol(circle)) ///
    (line men_line_y men_line_x, lcolor(black)  lpattern(dash) lwidth(medthick)) ///
    (line women_line_y women_line_x, lcolor(gs10) lpattern(dash) lwidth(medthick)), ///
    title("Study 1: MDE Across Sample Sizes") ///
    ytitle("Minimum Detectable Effect") ///
    xtitle("Total Sample Size (Gender Threat + Control)") ///
    xlabel(100(100)1050, grid) ///
    ylabel(, grid) ///
    legend(order( ///
        1 "Iraq War MDE" ///
        2 "Homosexuality MDE" ///
        3 "SUV Desirability MDE" ///
        4 "SUV Pay MDE" ///
        5 "Original Iraq War Effect" ///
        6 "Original Homosexuality Effect" ///
        7 "Original SUV Desire Effect" ///
        8 "Original SUV Pay Effect" ///
        9 "Replication Sample (Men)" ///
        10 "Replication Sample (Women)") ///
        ring(0) position(2) region(margin(l=-170 r=15)) size(small)) ///
    xline(1032, lpattern(dash) lcolor(black) lwidth(medthick)) ///
    xline(336, lpattern(dash) lcolor(gs10) lwidth(medthick)) ///
    graphregion(color(white))
graph export "FigureB3.pdf", as(pdf) replace

*************************
*************************
*********Study 2*********
*************************
*************************

*Now, we repeat the same exercise as above but for Study 2*

clear
set obs 42
gen n_total = 25 + (_n - 1) * 25
local nratio = 0.96571429	

local sd1 = 1.61
local sd2 = 0.96
local sd3 = 1.08
local sd4 = 1.23
local baseline1 = 3.26
local baseline2 = 3.41
local baseline3 = 5.04
local baseline4 = 2.96

foreach i in 1 2 3 4 {
    gen mde`i' = .
}

forvalues r = 1/42 {
    local N = n_total[`r']
    foreach i in 1 2 3 4 {
        local sd = `sd`i''
		local base = `baseline`i''
        power twomeans `base', n(`N') power(0.9) sd(`sd') nratio(`nratio') alpha(0.05)
        replace mde`i' = r(diff) in `r'
    }
}

* Observed effect points at Willer et al. men sample size*
gen obs_x1 = 40
gen obs_y1 = 1.10
gen obs_x2 = 40
gen obs_y2 = 0.36
gen obs_x3 = 40
gen obs_y3 = 0.02
gen obs_x4 = 40
gen obs_y4 = 0.73

* Total sample sizes for vertical lines
gen men_line_x = 1032
gen men_line_y = 0
gen men_line_y2 = 1

gen women_line_x = 336
gen women_line_y = 0
gen women_line_y2 = 1

twoway ///
    (line mde1 n_total, lcolor(blue%50)   lpattern(solid)     lwidth(medthick)) ///
    (line mde2 n_total, lcolor(red%50)    lpattern(dash)      lwidth(medthick)) ///
    (line mde3 n_total, lcolor(green%50)  lpattern(dot)       lwidth(medthick)) ///
    (line mde4 n_total, lcolor(orange%50) lpattern(longdash)  lwidth(medthick)) ///
    (scatter obs_y1 obs_x1, mcolor(blue)   msymbol(circle)) ///
    (scatter obs_y2 obs_x2, mcolor(red)    msymbol(circle)) ///
    (scatter obs_y3 obs_x3, mcolor(green)  msymbol(circle)) ///
    (scatter obs_y4 obs_x4, mcolor(orange) msymbol(circle)) ///
    (line men_line_y men_line_x, lcolor(black)  lpattern(dash) lwidth(medthick)) ///
    (line women_line_y women_line_x, lcolor(gs10) lpattern(dash) lwidth(medthick)), ///
    title("Study 2: MDE Across Sample Sizes") ///
    ytitle("Minimum Detectable Effect") ///
    xtitle("Total Sample Size (Gender Threat + Control)") ///
    xlabel(0(100)1050, grid) ///
    ylabel(, grid) ///
    legend(order( ///
        1 "Dominance Attitudes MDE" ///
        2 "Political Conservatism MDE" ///
        3 "System Justification MDE" ///
        4 "Traditionalism MDE" ///
        5 "Original Dom Att Effect" ///
        6 "Original Pol Cons Effect" ///
        7 "Original System Just Effect" ///
        8 "Original Traditionalism Effect" ///
        9 "Replication Sample (Men)" ///
        10 "Replication Sample (Women)") ///
        ring(0) position(2) region(margin(l=-170 r=15)) size(small)) ///
    xline(1032, lpattern(dash) lcolor(black) lwidth(medthick)) ///
    xline(336, lpattern(dash) lcolor(gs10) lwidth(medthick)) ///
    graphregion(color(white))
graph export "FigureB2.pdf", as(pdf) replace

*Now using our means and standard deviations*

clear
set obs 42
gen n_total = 25 + (_n - 1) * 25

local sd1 = 1.9333634
local sd2 = .8253713
local sd3 = 2.0080717
local sd4 = 1.636946
local baseline1 = 2.9379845
local baseline2 = 1.9483748
local baseline3 = 4.5238095
local baseline4 = 4.6939571
local nratio1 = 0.96317829
local nratio2 = 0.96175908
local nratio3 = 0.96190476
local nratio4 = 0.95711501

foreach i in 1 2 3 4 {
    gen mde`i' = .
}

forvalues r = 1/42 {
    local N = n_total[`r']
    foreach i in 1 2 3 4 {
        local sd = `sd`i''
		local base = `baseline`i''
		local nratio = `nratio`i''
        power twomeans `base', n(`N') power(0.9) sd(`sd') nratio(`nratio') alpha(0.05)
        replace mde`i' = r(diff) in `r'
    }
}

* Observed effect points at Willer et al. men sample size*
gen obs_x1 = 40
gen obs_y1 = 1.10
gen obs_x2 = 40
gen obs_y2 = 0.36
gen obs_x3 = 40
gen obs_y3 = 0.02
gen obs_x4 = 40
gen obs_y4 = 0.73

* Total sample sizes for vertical lines
gen men_line_x = 1032
gen men_line_y = 0
gen men_line_y2 = 1

gen women_line_x = 336
gen women_line_y = 0
gen women_line_y2 = 1

twoway ///
    (line mde1 n_total, lcolor(blue%50)   lpattern(solid)     lwidth(medthick)) ///
    (line mde2 n_total, lcolor(red%50)    lpattern(dash)      lwidth(medthick)) ///
    (line mde3 n_total, lcolor(green%50)  lpattern(dot)       lwidth(medthick)) ///
    (line mde4 n_total, lcolor(orange%50) lpattern(longdash)  lwidth(medthick)) ///
    (scatter obs_y1 obs_x1, mcolor(blue)   msymbol(circle)) ///
    (scatter obs_y2 obs_x2, mcolor(red)    msymbol(circle)) ///
    (scatter obs_y3 obs_x3, mcolor(green)  msymbol(circle)) ///
    (scatter obs_y4 obs_x4, mcolor(orange) msymbol(circle)) ///
    (line men_line_y men_line_x, lcolor(black)  lpattern(dash) lwidth(medthick)) ///
    (line women_line_y women_line_x, lcolor(gs10) lpattern(dash) lwidth(medthick)), ///
    title("Study 2: MDE Across Sample Sizes") ///
    ytitle("Minimum Detectable Effect") ///
    xtitle("Total Sample Size (Gender Threat + Control)") ///
    xlabel(0(100)1050, grid) ///
    ylabel(, grid) ///
    legend(order( ///
        1 "Dominance Attitudes MDE" ///
        2 "Political Conservatism MDE" ///
        3 "System Justification MDE" ///
        4 "Traditionalism MDE" ///
        5 "Original Dom Att Effect" ///
        6 "Original Pol Cons Effect" ///
        7 "Original System Just Effect" ///
        8 "Original Traditionalism Effect" ///
        9 "Replication Sample (Men)" ///
        10 "Replication Sample (Women)") ///
        ring(0) position(2) region(margin(l=-170 r=15)) size(small)) ///
    xline(1032, lpattern(dash) lcolor(black) lwidth(medthick)) ///
    xline(336, lpattern(dash) lcolor(gs10) lwidth(medthick)) ///
    graphregion(color(white))
graph export "FigureB4.pdf", as(pdf) replace
